{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets\n",
    "tf.set_random_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n",
      "Accuracy at step 0: 0.0717\n",
      "Accuracy at step 10: 0.4971\n",
      "Accuracy at step 20: 0.6672\n",
      "Accuracy at step 30: 0.7483\n",
      "Accuracy at step 40: 0.8211\n",
      "Accuracy at step 50: 0.8303\n",
      "Accuracy at step 60: 0.8532\n",
      "Accuracy at step 70: 0.86\n",
      "Accuracy at step 80: 0.8657\n",
      "Accuracy at step 90: 0.8788\n",
      "('Adding run metadata for', 99)\n",
      "Accuracy at step 100: 0.8785\n",
      "Accuracy at step 110: 0.8796\n",
      "Accuracy at step 120: 0.8864\n",
      "Accuracy at step 130: 0.8867\n",
      "Accuracy at step 140: 0.8855\n",
      "Accuracy at step 150: 0.8891\n",
      "Accuracy at step 160: 0.896\n",
      "Accuracy at step 170: 0.8942\n",
      "Accuracy at step 180: 0.897\n",
      "Accuracy at step 190: 0.8989\n",
      "('Adding run metadata for', 199)\n",
      "Accuracy at step 200: 0.9004\n",
      "Accuracy at step 210: 0.9012\n",
      "Accuracy at step 220: 0.9051\n",
      "Accuracy at step 230: 0.9047\n",
      "Accuracy at step 240: 0.9029\n",
      "Accuracy at step 250: 0.9037\n",
      "Accuracy at step 260: 0.9004\n",
      "Accuracy at step 270: 0.9087\n",
      "Accuracy at step 280: 0.9102\n",
      "Accuracy at step 290: 0.9098\n",
      "('Adding run metadata for', 299)\n",
      "Accuracy at step 300: 0.909\n",
      "Accuracy at step 310: 0.9073\n",
      "Accuracy at step 320: 0.9074\n",
      "Accuracy at step 330: 0.9111\n",
      "Accuracy at step 340: 0.9085\n",
      "Accuracy at step 350: 0.9107\n",
      "Accuracy at step 360: 0.9096\n",
      "Accuracy at step 370: 0.91\n",
      "Accuracy at step 380: 0.9127\n",
      "Accuracy at step 390: 0.9124\n",
      "('Adding run metadata for', 399)\n",
      "Accuracy at step 400: 0.9107\n",
      "Accuracy at step 410: 0.9149\n",
      "Accuracy at step 420: 0.9143\n",
      "Accuracy at step 430: 0.9124\n",
      "Accuracy at step 440: 0.9154\n",
      "Accuracy at step 450: 0.9142\n",
      "Accuracy at step 460: 0.9143\n",
      "Accuracy at step 470: 0.9113\n",
      "Accuracy at step 480: 0.9139\n",
      "Accuracy at step 490: 0.9151\n",
      "('Adding run metadata for', 499)\n",
      "Accuracy at step 500: 0.9163\n",
      "Accuracy at step 510: 0.9162\n",
      "Accuracy at step 520: 0.9151\n",
      "Accuracy at step 530: 0.9169\n",
      "Accuracy at step 540: 0.9171\n",
      "Accuracy at step 550: 0.9173\n",
      "Accuracy at step 560: 0.917\n",
      "Accuracy at step 570: 0.9168\n",
      "Accuracy at step 580: 0.9131\n",
      "Accuracy at step 590: 0.9146\n",
      "('Adding run metadata for', 599)\n",
      "Accuracy at step 600: 0.9155\n",
      "Accuracy at step 610: 0.9164\n",
      "Accuracy at step 620: 0.9165\n",
      "Accuracy at step 630: 0.9129\n",
      "Accuracy at step 640: 0.9189\n",
      "Accuracy at step 650: 0.9173\n",
      "Accuracy at step 660: 0.9174\n",
      "Accuracy at step 670: 0.9173\n",
      "Accuracy at step 680: 0.9193\n",
      "Accuracy at step 690: 0.9186\n",
      "('Adding run metadata for', 699)\n",
      "Accuracy at step 700: 0.9193\n",
      "Accuracy at step 710: 0.9196\n",
      "Accuracy at step 720: 0.9186\n",
      "Accuracy at step 730: 0.9178\n",
      "Accuracy at step 740: 0.9162\n",
      "Accuracy at step 750: 0.9194\n",
      "Accuracy at step 760: 0.9182\n",
      "Accuracy at step 770: 0.9193\n",
      "Accuracy at step 780: 0.9218\n",
      "Accuracy at step 790: 0.9211\n",
      "('Adding run metadata for', 799)\n",
      "Accuracy at step 800: 0.923\n",
      "Accuracy at step 810: 0.9235\n",
      "Accuracy at step 820: 0.9236\n",
      "Accuracy at step 830: 0.9228\n",
      "Accuracy at step 840: 0.9201\n",
      "Accuracy at step 850: 0.9208\n",
      "Accuracy at step 860: 0.92\n",
      "Accuracy at step 870: 0.9194\n",
      "Accuracy at step 880: 0.9208\n",
      "Accuracy at step 890: 0.9194\n",
      "('Adding run metadata for', 899)\n",
      "Accuracy at step 900: 0.9179\n",
      "Accuracy at step 910: 0.922\n",
      "Accuracy at step 920: 0.9208\n",
      "Accuracy at step 930: 0.921\n",
      "Accuracy at step 940: 0.9213\n",
      "Accuracy at step 950: 0.9228\n",
      "Accuracy at step 960: 0.9225\n",
      "Accuracy at step 970: 0.9225\n",
      "Accuracy at step 980: 0.9209\n",
      "Accuracy at step 990: 0.9233\n",
      "('Adding run metadata for', 999)\n",
      "Accuracy at step 1000: 0.9197\n"
     ]
    }
   ],
   "source": [
    "def train_1():\n",
    "    log_dir = \"log_1\"\n",
    "    mnist = read_data_sets(\"data\", one_hot=True, reshape=True, validation_size=0)\n",
    "    \n",
    "    if tf.gfile.Exists(log_dir):\n",
    "        tf.gfile.DeleteRecursively(log_dir)\n",
    "        tf.gfile.MakeDirs(log_dir)\n",
    "\n",
    "    sess = tf.InteractiveSession()\n",
    "    with tf.name_scope('input'):\n",
    "        x = tf.placeholder(tf.float32, [None, 784], name=\"x-input\")\n",
    "        #xx = tf.reshape(x, [-1, 784], name='xx-input')\n",
    "        y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')\n",
    "        \n",
    "        \n",
    "    def weight_variable(shape):\n",
    "        \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n",
    "        initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "        return tf.Variable(initial)\n",
    "    \n",
    "    def bias_variable(shape):\n",
    "        \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n",
    "        initial = tf.constant(0.1, shape=shape)\n",
    "        return tf.Variable(initial)\n",
    "    \n",
    "    def variable_summaries(var):\n",
    "        \"\"\"Attach a lot of summaries to a Tensor (for TensorBoard visualization).\"\"\"\n",
    "        with tf.name_scope('summaries'):\n",
    "            mean = tf.reduce_mean(var)\n",
    "            tf.summary.scalar('mean', mean)\n",
    "            with tf.name_scope('stddev'):\n",
    "                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))\n",
    "            tf.summary.scalar('stddev', stddev)\n",
    "            tf.summary.scalar('max', tf.reduce_max(var))\n",
    "            tf.summary.scalar('min', tf.reduce_min(var))\n",
    "            tf.summary.histogram('histogram', var)\n",
    "            \n",
    "    def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):\n",
    "        \"\"\"Reusable code for making a simple neural net layer.\n",
    "        It does a matrix multiply, bias add, and then uses relu to nonlinearize.\n",
    "        It also sets up name scoping so that the resultant graph is easy to read,\n",
    "        and adds a number of summary ops.\n",
    "        \"\"\"\n",
    "        # Adding a name scope ensures logical grouping of the layers in the graph.\n",
    "        with tf.name_scope(layer_name):\n",
    "            # This Variable will hold the state of the weights for the layer\n",
    "            with tf.name_scope('weights'):\n",
    "                weights = weight_variable([input_dim, output_dim])\n",
    "                variable_summaries(weights)\n",
    "            with tf.name_scope('biases'):\n",
    "                biases = bias_variable([output_dim])\n",
    "                variable_summaries(biases)\n",
    "            with tf.name_scope('Wx_plus_b'):\n",
    "                preactivate = tf.matmul(input_tensor, weights) + biases\n",
    "                tf.summary.histogram('pre_activations', preactivate)\n",
    "            activations = act(preactivate, name='activation')\n",
    "            tf.summary.histogram('activations', activations)\n",
    "            return activations\n",
    "        \n",
    "    y = nn_layer(x, 784, 10, 'layer1', act= tf.nn.softmax)\n",
    "    \n",
    "    with tf.name_scope('cross_entropy'):\n",
    "        # The raw formulation of cross-entropy,\n",
    "        #\n",
    "        # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)),\n",
    "        #                               reduction_indices=[1]))\n",
    "        #\n",
    "        # can be numerically unstable.\n",
    "        #\n",
    "        # So here we use tf.nn.softmax_cross_entropy_with_logits on the\n",
    "        # raw outputs of the nn_layer above, and then average across\n",
    "        # the batch.\n",
    "        diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    "        with tf.name_scope('total'):\n",
    "            cross_entropy = tf.reduce_mean(diff)\n",
    "    \n",
    "    tf.summary.scalar('cross_entropy', cross_entropy)\n",
    "    \n",
    "    with tf.name_scope('train'):\n",
    "        train_step = tf.train.AdamOptimizer(0.005).minimize(cross_entropy)\n",
    "    \n",
    "    with tf.name_scope('accuracy'):\n",
    "        with tf.name_scope('correct_prediction'):\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        with tf.name_scope('accuracy'):\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)\n",
    "    \n",
    "    merged = tf.summary.merge([accuracy_summary])\n",
    "    train_writer = tf.summary.FileWriter(log_dir + '/train', sess.graph)\n",
    "    test_writer = tf.summary.FileWriter(log_dir + '/test')\n",
    "    tf.global_variables_initializer().run()\n",
    "    \n",
    "    def feed_dict(train):\n",
    "        \"\"\"Make a TensorFlow feed_dict: maps data onto Tensor placeholders.\"\"\"\n",
    "        if train:\n",
    "            xs, ys = mnist.train.next_batch(100)\n",
    "        else:\n",
    "            xs, ys = mnist.test.images, mnist.test.labels\n",
    "        return {x: xs, y_: ys}\n",
    "    \n",
    "    max_steps = 1000+1\n",
    "    for i in range(max_steps):\n",
    "        if i % 10 == 0:  # Record summaries and test-set accuracy\n",
    "            summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False))\n",
    "            test_writer.add_summary(summary, i)\n",
    "            print('Accuracy at step %s: %s' % (i, acc))\n",
    "        else:  # Record train set summaries, and train\n",
    "            if i % 100 == 99:  # Record execution stats\n",
    "                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "                run_metadata = tf.RunMetadata()\n",
    "                summary, _ = sess.run([merged, train_step],\n",
    "                              feed_dict=feed_dict(True),\n",
    "                              options=run_options,\n",
    "                              run_metadata=run_metadata)\n",
    "                train_writer.add_run_metadata(run_metadata, 'step%03d' % i)\n",
    "                train_writer.add_summary(summary, i)\n",
    "                print('Adding run metadata for', i)\n",
    "            else:  # Record a summary\n",
    "                summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True))\n",
    "                train_writer.add_summary(summary, i)\n",
    "    train_writer.close()\n",
    "    test_writer.close()\n",
    "    \n",
    "#run train 1 function\n",
    "train_1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
